package com.xcn.siteinfo;

import java.io.*;

import net.sf.json.*;

import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;

import com.xcn.jdbc.*;
import com.xcn.lang.*;

/**
 * @author changmin
 */
public class Url_Writer
{
	protected static String DIRECTORY_URL = "http://directory.search.daum.net/site_list.daum?dirseq=0&ord=0&page=%s";

	protected final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
	protected final static String MYSQL_URL = "jdbc:mysql://127.0.0.1:3306";
	protected final static String MYSQL_DATABASE = "OCC";
	protected final static String MYSQL_USERID = "root";
	protected final static String MYSQL_USERPW = "root99";

	/**
	 * 
	 */
	public Url_Writer( )
	{
	}
	
	
	/**
	 * Directory ID Search
	 * 
	 * @param directoryName
	 * @return
	 */
	public String getDirectoryId ( String directoryName )
	{
		String result = "";
		String query = " SELECT  DIRECTORY_ID FROM DIRECTORY WHERE DIRECTORY_NAME = ? ";
		QueryBuilder _qb = new QueryBuilder ( MYSQL_DRIVER, MYSQL_URL, MYSQL_DATABASE, MYSQL_USERID, MYSQL_USERPW );
		try
		{
			JSONObject obj = _qb.getQueryResultObj ( query, directoryName );
			if ( obj != null )
				result = StringUtils.objFixNull ( obj.get ( "DIRECTORY_ID" ) );
		}
		catch ( Exception e )
		{
			e.printStackTrace ( );
		}
		return result;
	}

	/**
	 * Directory Insert
	 * 
	 * @param directoryName
	 * @param directoryPid
	 * @param comments
	 */
	public void insertDirectory ( String directoryName, String directoryPid, String comments, int level )
	{
		String query = " INSERT INTO DIRECTORY ( DIRECTORY_NAME, DIRECTORY_PID, COMMENTS, DIRECTORY_LEVEL ) VALUES ( ?, ?, ?, ? ) ";
		QueryBuilder _qb = new QueryBuilder ( MYSQL_DRIVER, MYSQL_URL, MYSQL_DATABASE, MYSQL_USERID, MYSQL_USERPW );
		try
		{
			_qb.executeUpdate ( query, directoryName, directoryPid, comments, String.valueOf ( level ) );
		}
		catch ( Exception e )
		{
			e.printStackTrace ( );
		}
	}
	
	/**
	 * Url Insert
	 * 
	 * @param url
	 * @param urlName
	 * @param urlDesc
	 * @param directoryId
	 */
	public void insertUrl( String url, String urlName, String urlDesc, String directoryId )
	{
		String check = " SELECT COUNT(*) AS CNT FROM URL_MST WHERE URL = ? AND DIRECTORY_ID = ? ";

		String query = " INSERT INTO URL_MST ( URL, URL_NAME, URL_DESC, DIRECTORY_ID ) VALUES ( ?, ?, ?, ? ) ";
		QueryBuilder _qb = new QueryBuilder ( MYSQL_DRIVER, MYSQL_URL, MYSQL_DATABASE, MYSQL_USERID, MYSQL_USERPW );
		try
		{
			if ( _qb.getQueryResultObj ( check, url, directoryId ).get ( "CNT" ).equals ( "0" ) )
			{
				_qb.executeUpdate ( query, url, urlName, urlDesc, directoryId );
			}
		}
		catch ( Exception e )
		{
			e.printStackTrace ( );
		}
	}
	

	
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main ( String[] args )
	{
		int start = StringUtils.objFixZero ( args[0] );
		int end = StringUtils.objFixZero ( args[1] );

		start = 1;
		end	= 70000;

		Url_Writer _writer = new Url_Writer ( );
		try
		{
			int failCnt = 0;
			for ( int pageNo = start ; pageNo < end ; pageNo++ )
			{
				System.out.println ( pageNo + " Page Searching..." );
				Document doc = null;
				
				if ( failCnt >= 5 )
					System.exit ( 0 );

				try
				{
					Connection con = Jsoup.connect ( String.format ( DIRECTORY_URL, String.valueOf ( pageNo ) ) );
					con.timeout ( 15000 );
					doc = con.get ( );
				}
				catch ( Exception e )
				{
					e.printStackTrace ( );
				}

				if ( doc == null )
				{
					failCnt++;
					Thread.sleep ( 3000 );
					--pageNo;
					continue;
				}

				Elements elements = (( org.jsoup.nodes.Element ) doc).getElementsByAttributeValue ( "class", "webTabImgTxt fl" );
				if ( elements.size ( ) == 0 )
				{
					elements = (( org.jsoup.nodes.Element ) doc).getElementsByAttributeValue ( "class", "webTabTxt" );
					if ( elements.size ( ) == 0 )
					{
						failCnt++;
						Thread.sleep ( 3000 );
						--pageNo;
						continue;
					}
				}

				for ( int i=0 ; i < elements.size ( ); i++ )
				{
					Element e = elements.get ( i );
					Elements webTabSite = e.getElementsByAttributeValue ( "class", "webTabSite" );
					Elements webTabDesc = e.getElementsByAttributeValue ( "class", "webTabDesc" );
					Elements urlLink = e.getElementsByAttributeValue ( "class", "urlLink" );
					Elements webTabCategory = e.getElementsByAttributeValue ( "class", "webTabCategory" );
					for ( int j = 0 ; j < webTabSite.size ( ) ; j++ )
					{
						String urlName = webTabSite.get ( j ).text ( );
						String urlDesc = webTabDesc.get ( j ).text ( );
						String url = urlLink.get ( j ).text ( );
						String category = "";
						if ( webTabCategory.size ( ) > 0 )
							category = webTabCategory.get ( j ).text ( ).replaceAll ( " > ", "\\|" ).trim ( );

						String directoryId = "-1";
	
						String[] directorys = category.split ( "\\|" );
						for ( int k = 0 ; k < directorys.length ; k++ )
						{
							directoryId = _writer.getDirectoryId ( directorys[k] );
							if ( directoryId.length ( ) == 0 )
							{
								String directoryPid = "-1";
								if ( k > 0 ) directoryPid = _writer.getDirectoryId ( directorys[k-1] );
	
								_writer.insertDirectory ( directorys[k], directoryPid, "", directorys.length );
							}

							directoryId = _writer.getDirectoryId ( directorys[k] );
						}
						//System.out.println ( String.format ( "%s\t%s", url, urlName ) );
						_writer.insertUrl ( url, urlName, urlDesc, directoryId );
					}
				}
				
				//int sleepTime = (int)(Math.random()*3)+1;
				//Thread.sleep ( sleepTime*1000 );
				failCnt = 0;
			}
		}
		catch ( Exception e )
		{
			e.printStackTrace ( );
		}

	}
}






























